#!/usr/bin/env python
# -*- coding:utf-8 -*-
##########Design Doc#############
#Initialize 

#data struct
#dict namecount
#dict ipcount

#for in range
	#get text
	#fetch data
	#statistic add name and ip
##########Design Doc#############

#Import urllib for HTML page fetching
import urllib2
#Import regular expression lib for Data extracting
import re

#Define Get Text function
def GetText (url):
	#Fetch whole HTML Page to String data
	try:
	    return urllib2.urlopen(url).read()
	#While URL is illegal
	except ValueError:
	    #Print "Bad link. Try to add 'http://' in the front."
	    print("Bad link. Try to add 'http://' in the front.")
	    #Quit the program
	    exit()
	#While BadStatusLine
	except httplib.BadStatusLine:
	    #Print "Oops! Error ocurs while connecting."
	    print("Oops! Error ocurs while connecting.")
	    #Quit the program
	    exit()


#post data source link
#urlbase = r'http://www.newsmth.net/bbscon.php?bid=936&id=14387'
urlbase = r'http://www.newsmth.net/bbscon.php?bid=936&id='

#Initialize file id number as last post
downlimit = 14493
upperlimit = 13636
count = 0
filecount = 0
idcount = 0
ipcount = 0
idbase = {}
ipbase = {}

#pattern for id and ip
idpattern = re.compile("\xb7\xa2\xd0\xc5\xc8\xcb: (\w+) \(.*\), \xd0\xc5\xc7\xf8")
ippattern = re.compile("\[FROM: (\d+\.\d+\.\d+\.\*)\]")

#get fetch stat
for fileidnum in range(upperlimit, downlimit):
#########init
	print(count + upperlimit)
	count += 1
	idkey = "none"
	ipkey = "none"
#########get text
	data = GetText(urlbase + str(fileidnum))
	#print data
#########fetch data
	#get id
	idmatchobj = idpattern.search(data)
	if idmatchobj:
		idkey = idmatchobj.group(1)
		if idbase.has_key(idkey):
			idbase[idkey] += 1
		else:
			idbase.setdefault(idkey,1)
		print idkey,
	#get ip
	ipmatchobj = ippattern.search(data)
	if ipmatchobj:
		ipkey = ipmatchobj.group(1)
		if ipbase.has_key(ipkey):
			ipbase[ipkey] += 1
			filecount += 1
		else:
			ipbase.setdefault(ipkey,1)
		print (ipkey)


#output result
#tempbase = dict((value,key) for key,value in idbase.iteritems())
#tempkeys = tempbase.keys()
#tempkeys.sort()
#for key in tempkeys:
#	idcount += 1
#	print ("%s = %d" % (tempbase[key], key))

#print ""

#tempbase = dict((value,key) for key,value in ipbase.iteritems())
#tempkeys = ipbase.keys()
#tempkeys.sort()
#for key in tempkeys:
#	ipcount += 1
#	print ("%s = %d" % (tempbase[key], key))
print ""
for key, value in idbase.items():
	idcount += 1
	print ("%s = %d" % (key, value))
print ""
for key, value in ipbase.items():
	ipcount += 1
	print ("%s = %d" % (key, value))
print ""
print count, "files posted", count-filecount, "files deleted", "\n", ipcount,"ips recorded",idcount,"ids recorded",

